# https://mp.weixin.qq.com/mp/profile_ext?
# action=getmsg
# &__biz=MjM5MTAwNDkwMA==  微信公众号的id(唯一固定标志)
# &f=json
# &offset=10  文章数据接口请求偏移量标志（从0开始），每次返回的json数据中会有下一次请求的offset，注意这里并不是按某些规则递增的
# &count=10   每次请求的数据量（亲测最多可以是10）
# &is_ok=1
# &scene=124
# &uin=MTY3MTE4MDE2NA%3D%3D
# &key=62ccc1e9be5d11651de942693fa546af630a7f5f2cb082168d706428dd09f63f2164b3abb2aeeccabe8414dbf087e3986847f48c39d70b544d3339d5fbf946b75d92186e69b1a6a0d9d4207135951fdf
# &pass_ticket=7HiFuB%2FuNFPN3QuCKKeqQCtChyelPPejafCn2tmLnOkZXdZeKtd2xJgKtENb95W7   可以理解是请求票据，而且隔一段时间后(大概几个小时)就会过期，这也是为什么微信公众号比较难按固定规则进行抓取的原因
# &wxtoken=
# &appmsg_token=1061_WlZVRTJ%252BgTq23lpR2w4Dd03NSJMqGP7M_fAL-Q~~   同样理解为非固定有过期策略的票据
# &x5=0
# &f=json

# https://mp.weixin.qq.com/mp/profile_ext?
# action=getmsg
# &__biz=MjM5MTAwNDkwMA==
# &f=json&offset=10
# &count=10
# &is_ok=1
# &scene=124
# &uin=MTY3MTE4MDE2NA%3D%3D
# &key=b1db0b7df0ed96b0fd9c0d80b3c5d737988014457bcaf45700e0b3af44473a3684a9aabec67bb3a58673728e2e0fb50e0eb8d4d825a741f4b1d95e3bef26d312cae3e43edb6110554b6ae3e0ee67d411
# &pass_ticket=DTTdl4Mg4ZQlSRebcbmJBqsK3wEsytx4w9rM7yJCCjcE3DEkxk2JXfTeEiIiwNCE
# &wxtoken=
# &appmsg_token=1061_7baRMXe5ZiUgPJS5ReOyzfsHFSrxwXQHifiuww~~
# &x5=0
# &f=json
import json
import ssl
import time
from datetime import datetime

import requests

from MySqlConn import Mysql


class WxDayEnglish(object):
    """微信公众号文章抓取爬虫"""

    def __init__(self, _biz, _pass_ticket, _app_msg_token, _cookie,_uin,_key, _offset=0):
        self.offset = _offset
        self.biz = _biz  # 公众号标志
        self.msg_token = _app_msg_token  # 票据(非固定)
        self.pass_ticket = _pass_ticket  # 票据(非固定)
        self.uin=_uin
        self.key = _key
        self.headers = {
            'Cookie': _cookie,  # Cookie(非固定)
            'Accept': '*/*',
            'Connection': 'keep-alive',
            'Accept-Encoding': 'gzip, deflate',
            'Host': 'mp.weixin.qq.com',
            'User-Agent': 'Mozilla/5.0 (Linux; Android 10; LYA-AL00 Build/HUAWEILYA-AL00L; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/107.0.5304.141 Mobile Safari/537.36 XWEB/5075 MMWEBSDK/20230405 MMWEBID/8677 MicroMessenger/8.0.35.2360(0x28002339) WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64'
        }
        # wx_mps = 'wxmps'  # 这里数据库、用户、密码一致(需替换成实际的)
        self.mySql =Mysql()

    def start(self):
        """请求获取公众号的文章接口"""

        offset = self.offset
        while True:
            api = 'https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz={0}&f=json&offset={1}' \
                  '&count=10&is_ok=1&scene=124&uin={2}&key={3}' \
                  '&pass_ticket={4}&wxtoken=&appmsg_token' \
                  '={5}&x5=0&f=json'.format(self.biz, offset,self.uin,self.key, self.pass_ticket, self.msg_token)
            print(api)
            resp = requests.get(api, headers=self.headers,verify=False).json()
            ret, status = resp.get('ret'), resp.get('errmsg')  # 状态信息
            if ret == 0 or status == 'ok':
                print('Crawl article: ' + api)
                offset = resp['next_offset']  # 下一次请求偏移量
                general_msg_list = resp['general_msg_list']
                msg_list = json.loads(general_msg_list)['list']  # 获取文章列表
                for msg in msg_list:
                    comm_msg_info = msg['comm_msg_info']  # 该数据是本次推送多篇文章公共的
                    msg_id = comm_msg_info['id']  # 文章id
                    post_time = datetime.fromtimestamp(comm_msg_info['datetime']).strftime("%Y-%m-%d %H:%M:%S")  # 发布时间
                    # msg_type = comm_msg_info['type']  # 文章类型
                    # msg_data = json.dumps(comm_msg_info, ensure_ascii=False)  # msg原数据

                    app_msg_ext_info = msg.get('app_msg_ext_info')  # article原数据
                    if app_msg_ext_info:
                        # 本次推送的首条文章
                        self._parse_articles(app_msg_ext_info, msg_id, post_time)
                        # 本次推送的其余文章
                        multi_app_msg_item_list = app_msg_ext_info.get('multi_app_msg_item_list')
                        if multi_app_msg_item_list:
                            for item in multi_app_msg_item_list:
                                msg_id = item['fileid']  # 文章id
                                if msg_id == 0:
                                    msg_id = int(time.time() * 1000)  # 设置唯一id,解决部分文章id=0出现唯一索引冲突的情况
                                self._parse_articles(item, msg_id, post_time)
                print('next offset is %d' % offset)
            else:
                print('Before break , Current offset is %d' % offset)
                break

    def _parse_articles(self, info, msg_id, post_time):
        """解析嵌套文章数据并保存入库"""

        title = info.get('title')  # 标题
        cover = info.get('cover')  # 封面图
        author = info.get('author')  # 作者
        digest = info.get('digest')  # 关键字
        source_url = info.get('source_url')  # 原文地址
        content_url = info.get('content_url')  # 微信地址
        print('插入数据，插入文章 %s' % title)
        self.mySql.insertOne(self._save_chapter(),(title,msg_id,content_url,cover,source_url,post_time,digest))



    @staticmethod
    def _save_chapter():
        sql = 'insert into chapter(title,fileid,content_url,cover,source_url,publishtime,digest) ' \
              'values(%s,%s,%s,%s,%s,%s,%s)'
        return sql

    @staticmethod
    def _timestamp2string(timeStamp):
        try:
            d = datetime.fromtimestamp(timeStamp)
            str1 = d.strftime("%Y-%m-%d %H:%M:%S")
            # 2015-08-28 16:43:37.283000'
            return str1
        except Exception as e:
            print
            e
            return ''


if __name__ == '__main__':
    biz = 'MjM5MTAwNDkwMA=='  # "公号标识"
    pass_ticket = '8gGl8df3WZ3sAKZya742s5rSE+E0FeA07mOjDxayYNMWip9Y2t0Fa+695L9Eu9zasfIU3JZfuvXpINmQlwEhzg=='  #请求票据，而且隔一段时间后(大概几个小时)就会过期，这也是为什么微信公众号比较难按固定规则进行抓取的原因
    app_msg_token = '1218_njDug1GM%2FnSpIHh8MCGuT_isx9AGbVjxIbqKRA~~' # 同样理解为非固定有过期策略的票据
    cookie = 'wxuin=651714222; devicetype=android-29; lang=zh_CN; version=28002339; rewardsn=; wxtokenkey=777; appmsg_token=1218_Jco6eMGeH6yUz7A9zdJycMvhUouNSH8YYGETo-ncBy60XFW-ob-P3797krmhKM0qEYyCmS9UvUW8axuY; pass_ticket=8gGl8df3WZ3sAKZya742s5rSE+E0FeA07mOjDxayYNMWip9Y2t0Fa+695L9Eu9zasfIU3JZfuvXpINmQlwEhzg==; wap_sid2=CK694bYCEooBeV9IR0dIVkM1SDl3dVY2bGJpSE8tUU4tTHpTbDdBcWt6WEd5akxKeThtSlh3WllVY21BTUVJUVZwNnFMend1b1V0NW9Kd3pfcDNDSWEwcHlmdFR2NG11SnVBVnkwWXFrU2dLRGhRc0FkbERvTTZpSzlEalpZSnJpRTRoUWhtdVQ3QnlLQVNBQUF+MMLhoaMGOA1AlU4='
    uin = '777'
    key = '777'
    # 以上信息不同公众号每次抓取都需要借助抓包工具做修改
    wxMps = WxDayEnglish(biz, pass_ticket, app_msg_token, cookie,uin,key)
    wxMps.start()  # 开始爬取文章

